{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The MIT License (MIT)\n",
    "\n",
    "# Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "\n",
    "# Permission is hereby granted, free of charge, to any person obtaining a copy of\n",
    "# this software and associated documentation files (the \"Software\"), to deal in\n",
    "# the Software without restriction, including without limitation the rights to\n",
    "# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n",
    "# the Software, and to permit persons to whom the Software is furnished to do so,\n",
    "# subject to the following conditions:\n",
    "\n",
    "# The above copyright notice and this permission notice shall be included in all\n",
    "# copies or substantial portions of the Software.\n",
    "\n",
    "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n",
    "# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n",
    "# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n",
    "# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n",
    "# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial: Feature Engineering for Recommender Systems\n",
    "\n",
    "# 3. Feature Engineering - Categorical\n",
    "\n",
    "## 3.4. Count Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('./data/train.parquet')\n",
    "df_valid = cudf.read_parquet('./data/valid.parquet')\n",
    "df_test = cudf.read_parquet('./data/test.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_test['brand'] = df_test['brand'].fillna('UNKNOWN')\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')\n",
    "df_test['cat_2'] = df_test['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 00:00:28 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>17800342</td>\n",
       "      <td>zeta</td>\n",
       "      <td>66.90</td>\n",
       "      <td>550465671</td>\n",
       "      <td>22650a62-2d9c-4151-9f41-2674ec6d32d5</td>\n",
       "      <td>0</td>\n",
       "      <td>computers</td>\n",
       "      <td>desktop</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 00:00:39 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 00:00:40 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 00:00:41 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 00:01:56 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004767</td>\n",
       "      <td>samsung</td>\n",
       "      <td>235.60</td>\n",
       "      <td>579970209</td>\n",
       "      <td>c6946211-ce70-4228-95ce-fd7fccdde63c</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:01:56</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                event_time event_type  product_id    brand   price    user_id  \\\n",
       "0  2019-12-01 00:00:28 UTC       cart    17800342     zeta   66.90  550465671   \n",
       "1  2019-12-01 00:00:39 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "2  2019-12-01 00:00:40 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "3  2019-12-01 00:00:41 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "4  2019-12-01 00:01:56 UTC       cart     1004767  samsung  235.60  579970209   \n",
       "\n",
       "                           user_session  target         cat_0        cat_1  \\\n",
       "0  22650a62-2d9c-4151-9f41-2674ec6d32d5       0     computers      desktop   \n",
       "1  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "2  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "3  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "4  c6946211-ce70-4228-95ce-fd7fccdde63c       0  construction        tools   \n",
       "\n",
       "     cat_2 cat_3            timestamp  ts_hour  ts_minute  ts_weekday  ts_day  \\\n",
       "0  UNKNOWN  <NA>  2019-12-01 00:00:28        0          0           6       1   \n",
       "1   vacuum  <NA>  2019-12-01 00:00:39        0          0           6       1   \n",
       "2   vacuum  <NA>  2019-12-01 00:00:40        0          0           6       1   \n",
       "3   vacuum  <NA>  2019-12-01 00:00:41        0          0           6       1   \n",
       "4    light  <NA>  2019-12-01 00:01:56        0          1           6       1   \n",
       "\n",
       "   ts_month  ts_year  \n",
       "0        12     2019  \n",
       "1        12     2019  \n",
       "2        12     2019  \n",
       "3        12     2019  \n",
       "4        12     2019  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = 'product_id'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Theory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>*Count Encoding (CE)*</b> calculates the frequency from one or more categorical features given the training dataset.<br><br>\n",
    "For example we can consider *Count Encoding* as the populiarity of an item or activity of an user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ce = df_train[cat].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1004767      317711\n",
       "1005115      251189\n",
       "1004856      227432\n",
       "4804056      224545\n",
       "1005100      180072\n",
       "              ...  \n",
       "100143590         1\n",
       "100143856         1\n",
       "100143867         1\n",
       "100144046         1\n",
       "100144443         1\n",
       "Name: product_id, Length: 164453, dtype: int32"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ce = ce.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>CE_product_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461352</th>\n",
       "      <td>2019-11-30 16:39:49 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1004873</td>\n",
       "      <td>samsung</td>\n",
       "      <td>346.83</td>\n",
       "      <td>561528294</td>\n",
       "      <td>67086842-ef36-4dc9-9a30-2aad4a7b191e</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 16:39:49</td>\n",
       "      <td>16</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>82398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461353</th>\n",
       "      <td>2019-11-30 16:39:55 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005115</td>\n",
       "      <td>apple</td>\n",
       "      <td>915.49</td>\n",
       "      <td>573793765</td>\n",
       "      <td>1966b5dc-bf01-40a9-b5d8-b2a0797be941</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 16:39:55</td>\n",
       "      <td>16</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>251189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461354</th>\n",
       "      <td>2019-11-30 16:39:56 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1004870</td>\n",
       "      <td>samsung</td>\n",
       "      <td>282.89</td>\n",
       "      <td>562742398</td>\n",
       "      <td>4817e492-3278-414d-9f6d-6a4305edc55c</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 16:39:56</td>\n",
       "      <td>16</td>\n",
       "      <td>39</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>105730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461355</th>\n",
       "      <td>2019-11-30 16:40:00 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005100</td>\n",
       "      <td>samsung</td>\n",
       "      <td>131.74</td>\n",
       "      <td>518629444</td>\n",
       "      <td>915b4d55-be00-4501-8eff-d40aec8de72c</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 16:40:00</td>\n",
       "      <td>16</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>180072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461356</th>\n",
       "      <td>2019-11-30 16:40:04 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005253</td>\n",
       "      <td>xiaomi</td>\n",
       "      <td>261.01</td>\n",
       "      <td>521716329</td>\n",
       "      <td>9598b75d-9342-4b56-aab6-9d8690f09a7a</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 16:40:04</td>\n",
       "      <td>16</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>30286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11461357 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       event_time event_type  product_id    brand   price  \\\n",
       "0         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "1         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "2         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "3         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "4         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "...                           ...        ...         ...      ...     ...   \n",
       "11461352  2019-11-30 16:39:49 UTC   purchase     1004873  samsung  346.83   \n",
       "11461353  2019-11-30 16:39:55 UTC   purchase     1005115    apple  915.49   \n",
       "11461354  2019-11-30 16:39:56 UTC   purchase     1004870  samsung  282.89   \n",
       "11461355  2019-11-30 16:40:00 UTC   purchase     1005100  samsung  131.74   \n",
       "11461356  2019-11-30 16:40:04 UTC   purchase     1005253   xiaomi  261.01   \n",
       "\n",
       "            user_id                          user_session  target  \\\n",
       "0         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "1         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "2         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "3         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "4         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "...             ...                                   ...     ...   \n",
       "11461352  561528294  67086842-ef36-4dc9-9a30-2aad4a7b191e       1   \n",
       "11461353  573793765  1966b5dc-bf01-40a9-b5d8-b2a0797be941       1   \n",
       "11461354  562742398  4817e492-3278-414d-9f6d-6a4305edc55c       1   \n",
       "11461355  518629444  915b4d55-be00-4501-8eff-d40aec8de72c       1   \n",
       "11461356  521716329  9598b75d-9342-4b56-aab6-9d8690f09a7a       1   \n",
       "\n",
       "                cat_0       cat_1    cat_2 cat_3            timestamp  \\\n",
       "0                <NA>        <NA>  UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "1                <NA>        <NA>  UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "2                <NA>        <NA>  UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "3                <NA>        <NA>  UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "4                <NA>        <NA>  UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "...               ...         ...      ...   ...                  ...   \n",
       "11461352  electronics  smartphone  UNKNOWN  <NA>  2019-11-30 16:39:49   \n",
       "11461353  electronics  smartphone  UNKNOWN  <NA>  2019-11-30 16:39:55   \n",
       "11461354  electronics  smartphone  UNKNOWN  <NA>  2019-11-30 16:39:56   \n",
       "11461355  electronics  smartphone  UNKNOWN  <NA>  2019-11-30 16:40:00   \n",
       "11461356  electronics  smartphone  UNKNOWN  <NA>  2019-11-30 16:40:04   \n",
       "\n",
       "          ts_hour  ts_minute  ts_weekday  ts_day  ts_month  ts_year  \\\n",
       "0              12         27           6       1        12     2019   \n",
       "1              12         27           6       1        12     2019   \n",
       "2              12         27           6       1        12     2019   \n",
       "3              12         27           6       1        12     2019   \n",
       "4              12         27           6       1        12     2019   \n",
       "...           ...        ...         ...     ...       ...      ...   \n",
       "11461352       16         39           5      30        11     2019   \n",
       "11461353       16         39           5      30        11     2019   \n",
       "11461354       16         39           5      30        11     2019   \n",
       "11461355       16         40           5      30        11     2019   \n",
       "11461356       16         40           5      30        11     2019   \n",
       "\n",
       "          CE_product_id  \n",
       "0                   881  \n",
       "1                   881  \n",
       "2                   881  \n",
       "3                   881  \n",
       "4                   881  \n",
       "...                 ...  \n",
       "11461352          82398  \n",
       "11461353         251189  \n",
       "11461354         105730  \n",
       "11461355         180072  \n",
       "11461356          30286  \n",
       "\n",
       "[11461357 rows x 20 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ce.columns = [cat, 'CE_' + cat]\n",
    "df_train.merge(ce, how='left', left_on=cat, right_on=cat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar, we can apply *Count Encoding* to a group of categorical features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ce = df_train[['cat_2', 'brand', 'target']].groupby(['cat_2', 'brand']).agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat_2</th>\n",
       "      <th>brand</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">UNKNOWN</th>\n",
       "      <th>UNKNOWN</th>\n",
       "      <td>521515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-case</th>\n",
       "      <td>367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-derma</th>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-elita</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-mega</th>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">winch</th>\n",
       "      <th>tutti</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vichy</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viteks</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>woodcraft</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yvesrocher</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11154 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    target\n",
       "                     count\n",
       "cat_2   brand             \n",
       "UNKNOWN UNKNOWN     521515\n",
       "        a-case         367\n",
       "        a-derma         57\n",
       "        a-elita         22\n",
       "        a-mega          25\n",
       "...                    ...\n",
       "winch   tutti           31\n",
       "        vichy           12\n",
       "        viteks          14\n",
       "        woodcraft       21\n",
       "        yvesrocher      11\n",
       "\n",
       "[11154 rows x 1 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>CE_cat_2_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 07:04:06 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>100008496</td>\n",
       "      <td>respect</td>\n",
       "      <td>95.24</td>\n",
       "      <td>529320958</td>\n",
       "      <td>ba3ecb00-0a81-480e-a35a-caee3ccc4b6e</td>\n",
       "      <td>0</td>\n",
       "      <td>apparel</td>\n",
       "      <td>shoes</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:04:06</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>28562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 07:04:07 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>5100816</td>\n",
       "      <td>xiaomi</td>\n",
       "      <td>32.18</td>\n",
       "      <td>554214170</td>\n",
       "      <td>06737067-47b5-4219-8bc7-b9c1fc017e74</td>\n",
       "      <td>0</td>\n",
       "      <td>apparel</td>\n",
       "      <td>shoes</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:04:07</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>483359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 07:04:09 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12704744</td>\n",
       "      <td>nokian</td>\n",
       "      <td>129.27</td>\n",
       "      <td>519798261</td>\n",
       "      <td>24472c68-b11e-4187-8603-cd71d14ddfba</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:04:09</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>40060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 07:04:09 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1005212</td>\n",
       "      <td>samsung</td>\n",
       "      <td>168.86</td>\n",
       "      <td>554551310</td>\n",
       "      <td>51d86227-dc05-4784-9a70-c6d45da1a17f</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:04:09</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>1212393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 07:04:10 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3601405</td>\n",
       "      <td>beko</td>\n",
       "      <td>180.16</td>\n",
       "      <td>513516750</td>\n",
       "      <td>fc2a04a8-8eaf-4727-8785-a6ae160ab9eb</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>washer</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:04:10</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>21073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461352</th>\n",
       "      <td>2019-11-30 17:06:47 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005115</td>\n",
       "      <td>apple</td>\n",
       "      <td>915.49</td>\n",
       "      <td>579789462</td>\n",
       "      <td>b5411c53-e888-4fc7-9042-5b647659a5ab</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 17:06:47</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>880314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461353</th>\n",
       "      <td>2019-11-30 17:06:50 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005161</td>\n",
       "      <td>xiaomi</td>\n",
       "      <td>191.56</td>\n",
       "      <td>531355728</td>\n",
       "      <td>36618eb8-4718-4abd-926f-b161f682d226</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 17:06:50</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>483359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461354</th>\n",
       "      <td>2019-11-30 17:06:53 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>11100284</td>\n",
       "      <td>scarlett</td>\n",
       "      <td>6.67</td>\n",
       "      <td>562924883</td>\n",
       "      <td>c05b8b4a-20a9-44a1-9a15-69bac0869c57</td>\n",
       "      <td>1</td>\n",
       "      <td>appliances</td>\n",
       "      <td>personal</td>\n",
       "      <td>scales</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 17:06:53</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>1310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461355</th>\n",
       "      <td>2019-11-30 17:06:53 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>2900852</td>\n",
       "      <td>dauscher</td>\n",
       "      <td>60.49</td>\n",
       "      <td>575788127</td>\n",
       "      <td>4efcc103-3d11-44fe-af39-cbccd523d8d1</td>\n",
       "      <td>1</td>\n",
       "      <td>appliances</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>microwave</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 17:06:53</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>1051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461356</th>\n",
       "      <td>2019-11-30 17:06:54 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>12711054</td>\n",
       "      <td>tunga</td>\n",
       "      <td>34.75</td>\n",
       "      <td>518767884</td>\n",
       "      <td>5ee565b9-7d98-4839-8215-026ff6aa6111</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 17:06:54</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>23337</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11461357 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       event_time event_type  product_id     brand   price  \\\n",
       "0         2019-12-01 07:04:06 UTC       cart   100008496   respect   95.24   \n",
       "1         2019-12-01 07:04:07 UTC       cart     5100816    xiaomi   32.18   \n",
       "2         2019-12-01 07:04:09 UTC       cart    12704744    nokian  129.27   \n",
       "3         2019-12-01 07:04:09 UTC       cart     1005212   samsung  168.86   \n",
       "4         2019-12-01 07:04:10 UTC       cart     3601405      beko  180.16   \n",
       "...                           ...        ...         ...       ...     ...   \n",
       "11461352  2019-11-30 17:06:47 UTC   purchase     1005115     apple  915.49   \n",
       "11461353  2019-11-30 17:06:50 UTC   purchase     1005161    xiaomi  191.56   \n",
       "11461354  2019-11-30 17:06:53 UTC   purchase    11100284  scarlett    6.67   \n",
       "11461355  2019-11-30 17:06:53 UTC   purchase     2900852  dauscher   60.49   \n",
       "11461356  2019-11-30 17:06:54 UTC   purchase    12711054     tunga   34.75   \n",
       "\n",
       "            user_id                          user_session  target  \\\n",
       "0         529320958  ba3ecb00-0a81-480e-a35a-caee3ccc4b6e       0   \n",
       "1         554214170  06737067-47b5-4219-8bc7-b9c1fc017e74       0   \n",
       "2         519798261  24472c68-b11e-4187-8603-cd71d14ddfba       0   \n",
       "3         554551310  51d86227-dc05-4784-9a70-c6d45da1a17f       0   \n",
       "4         513516750  fc2a04a8-8eaf-4727-8785-a6ae160ab9eb       0   \n",
       "...             ...                                   ...     ...   \n",
       "11461352  579789462  b5411c53-e888-4fc7-9042-5b647659a5ab       1   \n",
       "11461353  531355728  36618eb8-4718-4abd-926f-b161f682d226       1   \n",
       "11461354  562924883  c05b8b4a-20a9-44a1-9a15-69bac0869c57       1   \n",
       "11461355  575788127  4efcc103-3d11-44fe-af39-cbccd523d8d1       1   \n",
       "11461356  518767884  5ee565b9-7d98-4839-8215-026ff6aa6111       1   \n",
       "\n",
       "                 cat_0       cat_1      cat_2 cat_3            timestamp  \\\n",
       "0              apparel       shoes    UNKNOWN  <NA>  2019-12-01 07:04:06   \n",
       "1              apparel       shoes    UNKNOWN  <NA>  2019-12-01 07:04:07   \n",
       "2                 <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 07:04:09   \n",
       "3         construction       tools      light  <NA>  2019-12-01 07:04:09   \n",
       "4           appliances     kitchen     washer  <NA>  2019-12-01 07:04:10   \n",
       "...                ...         ...        ...   ...                  ...   \n",
       "11461352   electronics  smartphone    UNKNOWN  <NA>  2019-11-30 17:06:47   \n",
       "11461353   electronics  smartphone    UNKNOWN  <NA>  2019-11-30 17:06:50   \n",
       "11461354    appliances    personal     scales  <NA>  2019-11-30 17:06:53   \n",
       "11461355    appliances     kitchen  microwave  <NA>  2019-11-30 17:06:53   \n",
       "11461356          <NA>        <NA>    UNKNOWN  <NA>  2019-11-30 17:06:54   \n",
       "\n",
       "          ts_hour  ts_minute  ts_weekday  ts_day  ts_month  ts_year  \\\n",
       "0               7          4           6       1        12     2019   \n",
       "1               7          4           6       1        12     2019   \n",
       "2               7          4           6       1        12     2019   \n",
       "3               7          4           6       1        12     2019   \n",
       "4               7          4           6       1        12     2019   \n",
       "...           ...        ...         ...     ...       ...      ...   \n",
       "11461352       17          6           5      30        11     2019   \n",
       "11461353       17          6           5      30        11     2019   \n",
       "11461354       17          6           5      30        11     2019   \n",
       "11461355       17          6           5      30        11     2019   \n",
       "11461356       17          6           5      30        11     2019   \n",
       "\n",
       "          CE_cat_2_brand  \n",
       "0                  28562  \n",
       "1                 483359  \n",
       "2                  40060  \n",
       "3                1212393  \n",
       "4                  21073  \n",
       "...                  ...  \n",
       "11461352          880314  \n",
       "11461353          483359  \n",
       "11461354            1310  \n",
       "11461355            1051  \n",
       "11461356           23337  \n",
       "\n",
       "[11461357 rows x 20 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ce = ce.reset_index()\n",
    "ce.columns = ['cat_2', 'brand', 'CE_cat_2_brand']\n",
    "df_train.merge(ce, how='left', left_on=['cat_2', 'brand'], right_on=['cat_2', 'brand'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Count Encoding* creates a new feature, which can be used by the model for training. It groups categorical values based on the frequency together.<br><br>\n",
    "For example,<br>\n",
    "<li> users, which have only 1 interaction in the datasets, are encoded with 1. Instead of having 1 datapoint per user, now, the model can learn a behavior pattern of these users at once.<br>\n",
    "<li> products, which have many interactions in the datasets, are encoded with a high number. The model can learn to see them as top sellers and treat them, accordingly.<br><br>\n",
    "The advantage of Count Encoding is that the category values are grouped together based on behavior. Particularly in cases with only a few observation, a decision tree is not able to create a split and neural networks have only a few gradient descent updates for these values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<li> Count Encoding calculates frequency of categories<br>\n",
    "<li> The model is trained based on these frequencies<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "In competition, we could count encode the categories for the datasets in different ways:<br>\n",
    "<li> Count Encode the training dataset and apply it to the validation dataset<br>\n",
    "<li> Count Encode the training dataset and Count Encode the validataion dataset, separatly<br>\n",
    "<li> Merge the training dataset and validation dataset, Count Encode the concatenated dataset and apply to both datasets<br><br>\n",
    "Our focus is on industry applications, therefore only the first process is a valid real-world solution. We may can collect statistics as a stream and update the characteristic of our dataset, but it is probably cleaner to increase the training frequency of our recommender models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Practice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, it is your turn. Let's try to implement *Count Encoding* as a function. You can either use pandas, dask or cudf."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = 'user_id'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "### ToDo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution ###############"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution End ###########"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare the runtime between pandas and cuDF. The implementation depends only on the DataFrame object (calling function of the object) and does not require any pd / cuDF function. Therefore, we can use the same implementation and just use pandas.DataFrame and cuDF.DataFrame. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We restart the kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': True}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('./data/train.parquet')\n",
    "df_valid = cudf.read_parquet('./data/valid.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_pd = df_train.to_pandas()\n",
    "df_valid_pd = df_valid.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_encode(train, valid, col, gpu=True):\n",
    "    \"\"\"\n",
    "        train:  train dataset\n",
    "        valid:  validation dataset\n",
    "        col:    column which will be count encoded (in the example RESOURCE)\n",
    "    \"\"\"\n",
    "    # We keep the original order as cudf merge will not preserve the original order\n",
    "    if gpu:\n",
    "        train['org_sorting'] = cupy.arange(len(train), dtype=\"int32\")\n",
    "    else:\n",
    "        train['org_sorting'] = np.arange(len(train), dtype=\"int32\")\n",
    "    \n",
    "    train_tmp = train[col].value_counts().reset_index()\n",
    "    train_tmp.columns = [col,  'CE_' + col]\n",
    "    df_tmp = train[[col, 'org_sorting']].merge(train_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "    train['CE_' + col] = df_tmp['CE_' + col].fillna(0).values\n",
    "        \n",
    "    if gpu:\n",
    "        valid['org_sorting'] = cupy.arange(len(valid), dtype=\"int32\")\n",
    "    else:\n",
    "        valid['org_sorting'] = np.arange(len(valid), dtype=\"int32\")\n",
    "    df_tmp = valid[[col, 'org_sorting']].merge(train_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "    valid['CE_' + col] = df_tmp['CE_' + col].fillna(0).values\n",
    "    \n",
    "    valid = valid.drop('org_sorting', axis=1)\n",
    "    train = train.drop('org_sorting', axis=1)\n",
    "    return(train, valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.91 s, sys: 2.77 s, total: 9.68 s\n",
      "Wall time: 9.67 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_train_pd, df_valid_pd = count_encode(df_train_pd, df_valid_pd, 'user_id', gpu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 272 ms, sys: 272 ms, total: 544 ms\n",
      "Wall time: 542 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_train, df_valid = count_encode(df_train, df_valid, 'user_id', gpu=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our experiments, we achieve a speed up of 15.8x."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our implementation can be still improved. We will show a further optimized solution based on dask and dask_cudf."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We shutdown the kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': False}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
